#include "DictionaryProducer.hpp"
#include "StringFilter.hpp"
#include "SplitTool.hpp"
#include "StringSplit.hpp"
#include "Debug.hpp"
#include <stdio.h>
#include <string.h>
#include <dirent.h>
#include <time.h>
#include <unistd.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <sstream>
#include <vector>
#include <algorithm>
#include <memory>
#include <map>

namespace se {

DictionaryProducer::DictionaryProducer(const std::string &stop, const std::string &directory) {
    _files.push_back(stop);
    DIR *dir;
    struct dirent *ptr;
    if((dir = opendir(directory.c_str())) == NULL){
        perror("opendir");
        return;
    }
    while((ptr = readdir(dir)) != NULL){
        if(strcmp(ptr->d_name, ".") == 0 || strcmp(ptr->d_name, "..") == 0){
            continue;
        }
        else if(ptr->d_type == DT_REG){
            std::string strFile;
            strFile = directory;
            strFile += "/";
            strFile += ptr->d_name;
            _files.push_back(strFile);
            continue;
        }
        else{
            continue;
        }
    }
    closedir(dir);
    return;
}

DictionaryProducer::DictionaryProducer(const std::string &stop, const std::string &directory, SplitTool *splitTool)
    :_splitTool(splitTool){ 
        _files.push_back(stop);
        DIR *dir;
        struct dirent *ptr;
        if((dir = opendir(directory.c_str())) == NULL){
            perror("opendir");
            return;
        }
        while((ptr = readdir(dir)) != NULL){
            if(strcmp(ptr->d_name, ".") == 0 || strcmp(ptr->d_name, "..") == 0){
                continue;
            }
            else if(ptr->d_type == DT_REG){
                std::string strFile;
                strFile = directory;
                strFile += "/";
                strFile += ptr->d_name;
                _files.push_back(strFile);
                continue;
            }
            else{
                continue;
            }
        }
        closedir(dir);
        return;
    }

void DictionaryProducer::getFiles(){
    for(auto &str : _files){
        for(int i = str.size() - 1; i >= 0; i--){
            if(str.c_str()[i] == '\\'){
                _files[i] = '\0';
                break;
            }
        }
        std::cout << str << std::endl;
    }
    std::cout << std::endl;
}

void DictionaryProducer::buildEnDictionary(){
    /* std::cout << std::endl << _files[2] << std::endl; */
    for(size_t i = 1; i  < _files.size(); ++i){
        std::ifstream ifs(_files[i]);
        if(!ifs){
            std::cerr << "ifs is broken" << std::endl;
            return;
        }
        std::string line;
        readEnStopList();
        std::string word;
        while(ifs >> word){
            stringToLower(word);
            if(isAllDigit(word)){
                continue;
            }
            dealWord(word);
        }    
        this->dictionaryMapToVector();
        ifs.close();
    }
}


void DictionaryProducer::dictionaryMapToVector() {
    for (auto &element : this->_dictionary) {
        this->_dictionaryVector.push_back(element);
    }
}

void DictionaryProducer::buildCnDictionary(){
    readCnStopList();

    for(size_t i = 1; i < _files.size(); ++i){
        DEBUG_PRINT("parse file: %s", this->_files[i].c_str());
        std::ifstream ifs(_files[i]);
        if(!ifs){
            std::cerr << "ifs is broken" << std::endl;
            return;
        }
        std::string line;
        while(getline(ifs, line)){
            /* std::istringstream iss(line); */
            std::vector<std::string> words;

            _splitTool->cut(line, words);
            for(size_t i = 0; i < words.size(); ++i){
                if(words[i].size() < 3){
                    continue;
                }
                else{
                    if (_StopList.find(words[i]) == _StopList.end()) {
                        pushDictionary(words[i]);
                    }
                }
            }
        }
        ifs.close();
    }

    this->dictionaryMapToVector();
}
void DictionaryProducer::dealWord(const std::string &word){                                                           
    size_t idx=0,i=0;
    while(idx!=word.size()){    
        if(!isalpha(word[idx])){
            if(word.substr(i,idx-i)!=std::string())
            {   
                if(_StopList.count(word.substr(i,idx-i)) == 0){
                    pushDictionary(word.substr(i,idx-i));
                }
            }   
            i=idx+1;    
        }   
        idx++;
    }   
    if(word.substr(i,idx-i)!=std::string())
    {   
        if(_StopList.count(word.substr(i,idx-i)) == 0){
            pushDictionary(word.substr(i,idx-i));
        }
    }   

}

/* std::string DictionaryProducer::deleteWord(const std::string &word){ */
/*     for(size_t idx = 0; idx != word.size(); ++idx){ */
/*         if(!isalpha(word[idx])){ */
/*             return std::string(); */
/*         } */
/*     } */
/*     return word; */
/* } */

void DictionaryProducer::readEnStopList(){
    /* std::cout << std::endl << _files[0] << std::endl; */
    std::ifstream ifs2(_files[0]);
    if(!ifs2){
        std::cerr << "ifs2 is broken" << std::endl;
        return;
    }
    std::string str;
    while(ifs2 >> str){
        _StopList.insert(str);
    }
    ifs2.close();
}


void DictionaryProducer::readCnStopList(){
    /* std::cout << std::endl << _files[0] << std::endl; */
    std::ifstream ifs2(_files[0]);
    if(!ifs2){
        std::cerr << "ifs2 is broken" << std::endl;
        return;
    }
    std::string str;
    while(ifs2 >> str){
        _StopList.insert(str);
    }
    ifs2.close();
}

void DictionaryProducer::pushDictionary(const std::string &word){
    if(word == std::string()){
        return;
    }
    this->_dictionary[word]++;
}

void DictionaryProducer::storeDictionary(const std::string &filename){
    std::ofstream ofs(filename);
    if(!ofs){
        std::cerr << "ofs open failed" << std::endl;
        return;
    }

    nlohmann::json j(this->_dictionary);

    ofs<<j.dump(4);
    ofs.close();
}


void DictionaryProducer::createIndex(std::string src,std::string obj){   
    std::ifstream inputStream(src);
    nlohmann::json dictionaryJson;
    inputStream >> dictionaryJson;
    inputStream.close();

    std::vector<std::pair<std::string, int32_t>> dictionary(dictionaryJson.begin(), dictionaryJson.end());

    std::map<std::string, std::set<int64_t>> index;

    for (size_t i = 0; i < dictionary.size(); i++) {
        auto characterList = StringSplit::split(dictionary[i].first);
        for (auto element : *characterList) {
            index[element].insert(i);
        }
    }

    std::ofstream outputStream(obj);
    outputStream << nlohmann::json(index).dump(4);
    outputStream.close();
}

}  // namespace se

